#coding=UTF-8

import urllib2
import traceback
import codecs, sys
from HTMLParser import HTMLParser
from ictclas import *

gBaseUrl = 'http://zh.wikipedia.org'

gHeaders = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11',
    'Accept-Charset':'GBK,utf-8;q=0.7,*;q=0.3',
    'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4'
}

gIgnoreList = [
    '/wiki/File:', 
    '/wiki/Category:', 
    '/wiki/Project:', 
    '/wiki/Special:', 
    '/wiki/Wikipedia:', 
    '/wiki/Talk:', 
    '/wiki/Help:', 
    '/wiki/Portal:',
    '/wiki/Template:',
    '/wiki/Template_talk:'
]

class Entity(object):
    def __init__(self, word, url):
        self._word = word
        self._url = url
        self._wt = 0
        return
    
    def getWord(self):
        return self._word
    
    def getUrl(self):
        return self._url
    
    def setWeight(self, wt):
        self._wt = wt
    def getWeight(self):
        return self._wt
        
    def irrelevant(self):
        return self._wt < 20
            
    def __eq__(self, other):
        if other == None:
            return False
            
        return self._url.lower() == other._url.lower()

    def __hash__(self):
        return hash(self._url)
        
class MyParser(HTMLParser):
    
    def __init__(self, ownUrl = None):
        HTMLParser.__init__(self)
        self._entities = set([])
        self._content = None
        self._url = None
        self._ownUrl = ownUrl
        self._stop = False
        
    def getEntities(self):
        return self._entities

        
    def handle_starttag(self, tag, attrs):
        if self._stop == True:
            return
            
        if tag == 'a':
            hasFound = False
            if len(attrs) == 0: pass

            for (k, v) in attrs:
                if k == 'href' and (not willIgnore(v)):
                    if isEntityLink(v) and v != self._ownUrl:  
                        self._url = v
                        hasFound = True
                    else:
                        break
                        
                if k == 'title':
                    self._content = v
            
            if hasFound == True:        
                self._entities.add(Entity(self._content, self._url))
        
        if tag == 'table':
            for (k, v) in attrs:
                if k == 'class':
                    if v == 'navbox':
                        self._stop = True
        return

def willIgnore(url):
    for s in gIgnoreList:
        if url.startswith(s) == True:
            return True

    return False

def isEntityLink(url):
    return url[0:5] == '/wiki'

def fetchHtml(url, timeout):
    print('now_parsing@' + url)    
    req = urllib2.Request(url=gBaseUrl + url, headers=gHeaders)
    fp = urllib2.urlopen(req, None, timeout)
    html = fp.read()
    fp.close()
    return html
        
def walkEntry(e, depth, maxDepth=1, timeout=3000):
    allEntities = set()
    
    if e == None or depth > maxDepth or willIgnore(e.getUrl()):
        #print('reach_maxDepth:%s!!!\n'%e.getWord())
        return None
        
    try:
        url = e.getUrl()
        if not isEntityLink(url):
            return None

        html = fetchHtml(url, timeout)
        
        parser = MyParser(e.getUrl())
        parser.feed(unicode(html,'UTF-8'))
        parser.close()

        entities = parser.getEntities()
        cwa = Cwa()
        for e in entities:
            wa = cwa.analyzeWordAttr(e.getWord())
            wt = cwa.sumWeight(wa)
            e.setWeight(wt)
            
        while len(entities) > 200:
            minWtEt = None
            for e in entities:
                if minWtEt == None or e.getWeight() < minWtEt.getWeight():
                    minWtEt = e
            entities.remove(minWtEt)
            
        for e in entities:
            try:
                allEntities.add(e)
                #print(e.getWord().encode('GBK') + '=>' + str(e.getWeight()))
                innerEntities = walkEntry(e, depth + 1, maxDepth)
                if innerEntities != None and len(innerEntities) > 0:
                    allEntities.add(innerEntities)
            except Exception as e:
                print(e)
                continue
                
        return allEntities     
    except Exception,data:
        print('!!!FATAL_ERROR!!!')
        print traceback.print_exc()
        return None

if __name__ == '__main__':
    p = walkEntry(Entity(u'python','/wiki/%E6%AF%9B%E6%B3%BD%E4%B8%9C'), 1, 1)
    q = walkEntry(Entity(u'csharp','/wiki/%E6%96%AF%E5%A4%A7%E6%9E%97'), 1, 1)
    print('p=%d;q=%d'%(len(p),len(q)))
    x = p.intersection(q)
    
    print('---------------')
    for e in x:
        print(e.getWord())